# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate:
# Message:
import random

import requests
import hashlib
import redis
import time
from lxml import etree
import json
from collections import OrderedDict
import pymysql
import os
import sys
# 解决编码问题
reload(sys)
sys.setdefaultencoding('utf-8')

class BmwPartPriceForPid(object):

    def __init__(self, db_config):

        self.db_config = db_config
        self.detail_url = "https://www.dwp.bmw-brilliance.cn/sap(bD16aCZjPTEwMA==)/bc/bsp/bmw/gis_tp_par_dev/spp05_material_details.htm?if_material={}"
        self.base_path = '/Users/duanmengdi/Desktop/workspace/bmw_part_price_html/'  # 存储html文件的路径
        self.total_num = 0  # 全局计数变量
        self.sleep_num = 0  # 控制抓取速度计数变量
        self.continue_num = 0  # 断点续爬计数

        self.my_headers = {
            "Host": "www.dwp.bmw-brilliance.cn",
            "Authorization": "Basic NDA0MjlwYXI6MXEydzNlNHI=",
            "Origin": "https://www.dwp.bmw-brilliance.cn",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
            "Referer": "https://www.dwp.bmw-brilliance.cn/sap(bD16aCZjPTEwMA==)/bc/bsp/bmw/gis_tp_par_dev/spp05_stock_check.htm",
        }

        self.my_cookies = {
            'sap-contextid': "SID%3aANON%3apcqpdi33_CQP_33%3aal--7EXL1z9yGpu2bQxQot5jos7rfMjhERkPy_uy-NEW",
            "sap-appcontext": "c2FwLXNlc3Npb25pZD1TSUQlM2FBTk9OJTNhcGNxcGRpMzNfQ1FQXzMzJTNhYWwtLUVCenVJZldDVnU2MW1CNkpTaEVrVUQzc2ZNaFlDaGxiV05wTy1BVFQ%3d",
            "SelectedMaterials": "",
            "Order_Selected_materials": "",
            "sap-usercontext": "sap-client=100",
            "BIGipServercqpcn.bmwbrill.cn_pool_8443": "3528863392.64288.0000",
            "MYSAPSSO2": "AjQxMDMBABg0ADAANAAyADkAUABBAFIAIAAgACAAIAACAAYxADAAMAADABBDAFEAUAAgACAAIAAgACAABAAYMgAwADEAOAAwADcAMgA2ADAAMgA1ADEABQAEAAAACAYAAlgACQACMQD%2fAVYwggFSBgkqhkiG9w0BBwKgggFDMIIBPwIBATELMAkGBSsOAwIaBQAwCwYJKoZIhvcNAQcBMYIBHjCCARoCAQEwbzBkMQswCQYDVQQGEwJERTEcMBoGA1UEChMTU0FQIFRydXN0IENvbW11bml0eTETMBEGA1UECxMKU0FQIFdlYiBBUzEUMBIGA1UECxMLSTAwMjA0NzQyNjExDDAKBgNVBAMTA0NRUAIHIBUJAglTWTAJBgUrDgMCGgUAoF0wGAYJKoZIhvcNAQkDMQsGCSqGSIb3DQEHATAcBgkqhkiG9w0BCQUxDxcNMTgwNzI2MDI1MTM3WjAjBgkqhkiG9w0BCQQxFgQUAWscGja36SWqroDFcHJ%21h%2fhDeLgwCQYHKoZIzjgEAwQvMC0CFAjCrM3Sx2D9wvczUSbSqkkZ%21eRSAhUAoLDaaa4v642bJ3CUL0fcAgzPY3Y%3d",
            "lf_actionspage": "spp05_order_create.htm%3foninputprocessing(update)",
            "ordtype_create": "",
            "date_create": "",
            "po_create": "",
            "sppbasket": "",
            "sap-ssolist": "O3I9cGNxcGRpMzNfQ1FQXzMzXzEwMF81NkU5MEFFNURDN0JENkU4Qjc3NzM4NDhBM0E4NUY0NzE0RTRBRjQz",
            "SAP_SESSIONID_CQP_100": "HgiPzotMLD1a14Riv78EtSQAvhmTmxHog24AFj4uVso%3d",
        }

        self.post_data = {
            "onInputProcessing": "atp_all",
            "contentClip": "",
            "req_date": "27.07.2018",
            "material": "{}",  # 查询的pid
            "x": "7",
            "y": "7",
        }


    def getmd5(self, str):
        md5 = hashlib.md5()
        md5.update(str)
        return md5.hexdigest()

    def get_conn(self, dbconfig_dict):
        conn = pymysql.connect(**dbconfig_dict)
        return conn

    def get_cursor(self, conn, type='stream'):
        if type == 'stream':
            return conn.cursor(pymysql.cursors.SSCursor)  # 返回流式游标,查询大量数据时不占用内存(返回数据形式是元组)
        elif type == 'dict':
            return conn.cursor(pymysql.cursors.DictCursor)  # 返回字典形式游标,查询出的数据以字典形式返回
        else:
            raise Exception("cursor type error")

    def parse_url(self, url, method, post_data=None, headers=None, tofile=True, **kwargs):
        requests.packages.urllib3.disable_warnings()  # 因为把ssl安全验证关了,所以不让代码报不安全的https请求警告
        pid = kwargs.get("local_pid", "")
        if method == 'GET':
            data = "GET" + url + pid  # 拼接data
            md5 = self.getmd5(data)  # 获取此次请求电子指纹
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "GET for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res.decode('utf8')
            else:  # 如果路径不存在, 从web端请求
                print "GET for web"
                try:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.my_cookies, timeout=30,
                                            allow_redirects=False)
                except Exception as e:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.my_cookies, timeout=30,
                                            allow_redirects=False)

                # 获取cookies, 并更新self.cookies
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.my_cookies[k] = cookies[k]

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                    # print "返回正经的response"
                else:
                    # print "返回不正经的None"
                    response = None

        elif method == "POST":
            data = "POST" + url + json.dumps(post_data)
            md5 = self.getmd5(data)
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "POST for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res.decode('utf8')
            else:
                print "POST for web"
                try:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers, cookies=self.my_cookies, timeout=30,
                                             allow_redirects=False)
                except Exception as e:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers, cookies=self.my_cookies, timeout=30,
                                             allow_redirects=False)

                # 获取cookies, 并更新self.cookies
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.my_cookies[k] = cookies[k]

                if response:
                    # cookies = response.cookies.get_dict()  # 可以在此获取cookies
                    # print cookies
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")

                else:
                    response = None
        return response

    def save2db(self, data_dict, curs):
        pid = data_dict['pid'].decode('utf8')
        brandCode = data_dict['brandCode'].decode('utf8')
        price = data_dict['price'].decode('utf8')
        stock_price = data_dict['stock_price'].decode('utf8')
        rush_price = data_dict['rush_price'].decode('utf8')

        # 拼接sql语句
        sql = u'''INSERT INTO bmw_part_4s_price (`pid`,`brandCode`,`price`,`stock_price`,`rush_price`) VALUES ('%s','%s','%s','%s','%s')''' % (pid, brandCode, price, stock_price, rush_price)

        try:
            curs.execute(sql)
        except Exception as e:
            print e
            print sql
            exit()
        self.total_num += 1  # 数据总量计数

    def run(self):
        # 首先从数据库读取pid出来
        # 首先获取数据库链接和游标
        conn1 = self.get_conn(dbconfig_dict=self.db_config)
        conn2 = self.get_conn(dbconfig_dict=self.db_config)  # 用于存储
        curs1 = self.get_cursor(conn=conn1)
        curs2 = self.get_cursor(conn=conn2)  # 用于存储
        # 获取redis连接 实现断点续爬
        r = redis.Redis(host='127.0.0.1', port=6379, db=0)

        # 查询语句
        sql = u'''select pid from pid_brandCode'''
        curs1.execute(sql)
        for data in curs1:
            pid = data[0]
            # if r.sismember('pid_set', pid):
            #     print "已经抓取过, 跳过"
            #     continue

            # 判断连续抓取次数是否达到临界点, 临界点为10, 每抓10次随机sleep3-5秒
            # if self.sleep_num >= 10:
            #     sleep_time = random.randint(3,5)
            #     print "连续抓取10次, 间歇%d秒" % sleep_time
            #     time.sleep(sleep_time)
            #     self.sleep_num = 0  # 间歇完毕,重置为0
            # self.sleep_num += 1
            try:
                search_pid = str("%018d" % int(pid))
            except:
                search_pid = str(pid)

            detail_url = self.detail_url.format(search_pid)
            # print detail_url

            try:
                detail_response = self.parse_url(url=detail_url, method="GET", headers=self.my_headers, local_pid=pid)
                if detail_response:
                    detail_html = etree.HTML(detail_response)
                else:
                    print "detail_reaponse is None type"
                    exit()

                price = detail_html.xpath(u'//td[contains(text(),"零售价")]/following-sibling::td[1]/text()')
                stock_price = detail_html.xpath(u'//td[contains(text(),"净值 (Stock)")]/following-sibling::td[1]/text()')
                rush_price = detail_html.xpath(u'//td[contains(text(),"净值 (Rush)")]/following-sibling::td[1]/text()')

                price = price[0] if len(price)>0 else ""
                stock_price = stock_price[0] if len(stock_price)>0 else ""
                rush_price = rush_price[0] if len(rush_price)>0 else ""
                # print price, stock_price, rush_price

                # 数据清洗
                price = float(price.replace(' ', '').replace(',', '').replace('.', '').replace('RMB', '').replace('\n', ''))
                stock_price = float(stock_price.replace(' ', '').replace(',', '').replace('.', '').replace('RMB', '').replace('\n', ''))
                rush_price = float(rush_price.replace(' ', '').replace(',', '').replace('.', '').replace('RMB', '').replace('\n', ''))

                # 数据过滤条件
                if price == 0.0:
                    with open('./no_selected_pid.txt', 'a') as f:
                        f.write(pid)
                        f.write('\n')
                    continue

                # 拼装最终字典
                last_data = dict(
                    pid=pid,
                    brandCode='bmw',
                    price=str(price),
                    stock_price=str(stock_price),
                    rush_price=str(rush_price)
                )
                print last_data
                self.save2db(last_data, curs2)

                #  把pid加入redis中,下次启动实现断点续爬
                r.sadd('pid_set', pid)
                r.save()

                if self.total_num % 100 == 0:
                    try:
                        conn2.commit()
                        print "Commit insert cache, current num: {}".format(self.total_num)
                    except Exception as e:
                        raise e
            except Exception as e:
                conn2.commit()
                raise e

        conn2.commit()
        print "最终提交, 总数据量: ", self.total_num

        curs1.close()
        curs2.close()
        conn1.close()
        conn2.close()


if __name__ == '__main__':
    db_config = dict(
        host="127.0.0.1",
        port=3306,
        user="root",
        passwd="123456",
        db="my_data",
        charset="utf8"
    )
    tools = BmwPartPriceForPid(db_config=db_config)
    tools.run()